🌎 rns.kin.earth git 🌍
Commit 803b54895186d0396c71dcee57a640351ca9721e
Parents : 67a45ac
Author : Mark Qvist <bc7291552be7a58f361522990465165c>
Signature : T66BB85Valid, signed by author
Date : 2026-08-27T14:56:47+02:00
Allow loading compiled modules
Changes
10 files changed, 260 insertions(+), 21 deletions(-)
Diff
diff --git a/CRNS/__init__.py b/CRNS/__init__.py
index 49340625..de6c7474 100644
--- a/CRNS/__init__.py
+++ b/CRNS/__init__.py
@@ -1,12 +1,232 @@
compiling = False
noticed = False
notice_delay = 0.3
-import time
+import os
import sys
+import time
import threading
from importlib.util import find_spec
if find_spec("pyximport") and find_spec("cython"):
- import pyximport; pyxloader = pyximport.install(pyimport=True, language_level=3)[1]
+ import json
+ from importlib.abc import MetaPathFinder
+ from importlib.machinery import ExtensionFileLoader, SourceFileLoader
+ from importlib.machinery import EXTENSION_SUFFIXES
+ from importlib.util import spec_from_file_location
+ from distutils.extension import Extension
+
+ from pyximport import pyxbuild
+
+ class _RNSImportFinder(MetaPathFinder):
+ """A cythonizing import hook for the RNS package tree.
+
+ pyximport's own ``.py`` hook is unsuitable for this: it only
+ resolves top-level module names (so every sub-module import
+ misses the file), and it disables itself after compiling a
+ single module.
+
+ This finder compiles RNS source modules into a mirror tree
+ under ``~/.pyxbld/rnslib/`` that exactly mirrors the source
+ package layout, so that:
+
+ * ``os.path.dirname(__file__)`` of a compiled package
+ initialiser equals that package's own mirror directory,
+ keeping the ``__all__`` globs in package ``__init__``
+ modules correct when the modules are compiled.
+ * sub-module resolution through ``__path__`` finds the
+ compiled extensions in the mirror.
+ * each loaded module is a C extension (``.so``/``.pyd``).
+
+ The entire RNS source tree is compiled up front (on first use,
+ and incrementally on later uses), so the mirror is complete
+ before any RNS module is imported and imported modules can be
+ loaded straight from it.
+
+ Third-party and standard-library modules are left untouched,
+ and the ``RNS.vendor`` subtree is deliberately imported from
+ source. Modules that cannot be compiled fall back to the
+ interpreted source module with a warning.
+ """
+
+ def __init__(self):
+ self.build_dir = os.path.join(os.path.expanduser("~"), ".pyxbld")
+ self.mirror_root = os.path.join(self.build_dir, "rnslib")
+ self.source_root = self._find_source_root()
+ self.abi_suffix = EXTENSION_SUFFIXES[0]
+ self.unbuildable = {}
+ self.unbuildable_path = os.path.join(self.build_dir, "unbuildable.json")
+ try:
+ with open(self.unbuildable_path, "r") as fh:
+ self.unbuildable = json.load(fh)
+ except Exception:
+ self.unbuildable = {}
+
+ def _save_unbuildable(self):
+ try:
+ os.makedirs(os.path.dirname(self.unbuildable_path), exist_ok=True)
+ with open(self.unbuildable_path, "w") as fh:
+ json.dump(self.unbuildable, fh, indent=1)
+ except Exception:
+ pass
+
+ def _mark_unbuildable(self, fullname, source):
+ try:
+ self.unbuildable[fullname] = int(os.path.getmtime(source))
+ self._save_unbuildable()
+ except Exception:
+ pass
+
+ def _unmark_buildable(self, fullname):
+ if fullname in self.unbuildable:
+ del self.unbuildable[fullname]
+ self._save_unbuildable()
+
+ def _is_unbuildable(self, fullname, source):
+ if fullname not in self.unbuildable:
+ return False
+ try:
+ if int(os.path.getmtime(source)) == self.unbuildable[fullname]:
+ return True
+ except Exception:
+ pass
+ self.unbuildable.pop(fullname, None)
+ return False
+
+ def _find_source_root(self):
+ here = os.path.dirname(os.path.abspath(__file__))
+ candidate = os.path.abspath(os.path.join(here, "..", "RNS", "__init__.py"))
+ if os.path.exists(candidate):
+ return os.path.dirname(os.path.dirname(candidate))
+ for entry in [os.getcwd()] + list(sys.path):
+ if not entry:
+ continue
+ candidate = os.path.join(entry, "RNS", "__init__.py")
+ if os.path.exists(candidate):
+ return os.path.dirname(candidate)
+ raise RuntimeError("CRNS could not locate the RNS source tree")
+
+ def _all_source_modules(self):
+ rns_root = os.path.join(self.source_root, "RNS")
+ for root, dirs, files in os.walk(rns_root):
+ dirs[:] = [d for d in dirs if d not in ("__pycache__", "vendor")]
+ rel = os.path.relpath(root, self.source_root).replace(os.sep, ".")
+ for fn in sorted(files):
+ if not fn.endswith(".py"):
+ continue
+ if fn == "__init__.py":
+ fullname = rel
+ else:
+ fullname = rel + "." + fn[:-3]
+ yield fullname
+
+ def _mirror_target(self, fullname, is_package):
+ rel = fullname.split(".")
+ basename = rel[-1]
+ if is_package:
+ return os.path.join(self.mirror_root, *rel, "__init__" + self.abi_suffix)
+ return os.path.join(self.mirror_root, *rel[:-1], basename + self.abi_suffix)
+
+ def _source(self, fullname):
+ rel = fullname.split(".")
+ package_source = os.path.join(self.source_root, *rel, "__init__.py")
+ if os.path.exists(package_source):
+ return package_source, True
+ module_source = os.path.join(self.source_root, *rel) + ".py"
+ if os.path.exists(module_source):
+ return module_source, False
+ return None, None
+
+ def _compile(self, fullname, source):
+ basename = fullname.rsplit(".", 1)[-1]
+ extension = Extension(name=basename, sources=[source])
+ so_path = pyxbuild.pyx_to_dll(
+ source,
+ ext=extension,
+ build_in_temp=True,
+ pyxbuild_dir=self.build_dir,
+ )
+ abi_suffix = os.path.basename(so_path)[len(basename):]
+ return so_path, abi_suffix
+
+ def _ensure_compiled(self, fullname, source, is_package):
+ target = self._mirror_target(fullname, is_package)
+ if os.path.exists(target):
+ return target
+ so_path, abi_suffix = self._compile(fullname, source)
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ os.replace(so_path, target)
+ return target
+
+ def precompile(self):
+ if not os.path.isdir(self.mirror_root):
+ os.makedirs(self.mirror_root, exist_ok=True)
+ for fullname in self._all_source_modules():
+ source, is_package = self._source(fullname)
+ if source is None:
+ continue
+ if os.path.exists(self._mirror_target(fullname, is_package)):
+ self._unmark_buildable(fullname)
+ continue
+ if self._is_unbuildable(fullname, source):
+ continue
+ try:
+ self._ensure_compiled(fullname, source, is_package)
+ self._unmark_buildable(fullname)
+ except Exception as e:
+ self._mark_unbuildable(fullname, source)
+ print(f"CRNS: could not compile {fullname}, "
+ f"using interpreted module: {e}", file=sys.stderr)
+
+ def _spec(self, fullname, source, is_package, compile_attempted):
+ if fullname == "RNS.vendor" or fullname.startswith("RNS.vendor."):
+ rel = fullname.split(".")
+ locations = [os.path.join(self.source_root, *rel)] if is_package else None
+ return spec_from_file_location(
+ fullname, source,
+ loader=SourceFileLoader(fullname, source),
+ submodule_search_locations=locations,
+ )
+
+ if compile_attempted:
+ try:
+ target = self._ensure_compiled(fullname, source, is_package)
+ self._unmark_buildable(fullname)
+ if is_package:
+ locations = [os.path.dirname(target)]
+ else:
+ locations = None
+ return spec_from_file_location(
+ fullname, target,
+ loader=ExtensionFileLoader(fullname, target),
+ submodule_search_locations=locations,
+ )
+ except Exception as e:
+ self._mark_unbuildable(fullname, source)
+ print(f"CRNS: could not compile {fullname}, "
+ f"using interpreted module: {e}", file=sys.stderr)
+
+ rel = fullname.split(".")
+ locations = [os.path.join(self.source_root, *rel)] if is_package else None
+ return spec_from_file_location(
+ fullname, source,
+ loader=SourceFileLoader(fullname, source),
+ submodule_search_locations=locations,
+ )
+
+ def find_spec(self, fullname, path, target=None):
+ if not isinstance(fullname, str):
+ return None
+ if not (fullname == "RNS" or fullname.startswith("RNS.")):
+ return None
+
+ source, is_package = self._source(fullname)
+ if source is None:
+ return None
+
+ return self._spec(fullname, source, is_package,
+ compile_attempted=not self._is_unbuildable(fullname, source))
+
+ _RNSFinder = _RNSImportFinder()
+ sys.meta_path.insert(0, _RNSFinder)
def notice_job():
global noticed
@@ -22,5 +242,6 @@ def notice_job():
compiling = True
threading.Thread(target=notice_job, daemon=True).start()
+_RNSFinder.precompile()
import RNS; compiling = False
-if noticed: print("Done."); sys.stdout.flush()
\ No newline at end of file
+if noticed: print("Done."); sys.stdout.flush()
diff --git a/RNS/Cryptography/__init__.py b/RNS/Cryptography/__init__.py
index 925221de..0ccc23d7 100644
--- a/RNS/Cryptography/__init__.py
+++ b/RNS/Cryptography/__init__.py
@@ -52,5 +52,7 @@ elif cp.PROVIDER == cp.PROVIDER_PYCA:
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
\ No newline at end of file
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
\ No newline at end of file
diff --git a/RNS/Interfaces/Android/__init__.py b/RNS/Interfaces/Android/__init__.py
index a9a74ff3..454b31ff 100644
--- a/RNS/Interfaces/Android/__init__.py
+++ b/RNS/Interfaces/Android/__init__.py
@@ -33,5 +33,7 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
\ No newline at end of file
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
\ No newline at end of file
diff --git a/RNS/Interfaces/__init__.py b/RNS/Interfaces/__init__.py
index 9a11d8fd..56a541bf 100755
--- a/RNS/Interfaces/__init__.py
+++ b/RNS/Interfaces/__init__.py
@@ -36,5 +36,7 @@ import RNS.Interfaces.util.netinfo as netinfo
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
\ No newline at end of file
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
\ No newline at end of file
diff --git a/RNS/Interfaces/util/__init__.py b/RNS/Interfaces/util/__init__.py
index 72302cb5..e90e61ad 100644
--- a/RNS/Interfaces/util/__init__.py
+++ b/RNS/Interfaces/util/__init__.py
@@ -3,5 +3,7 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
diff --git a/RNS/Utilities/__init__.py b/RNS/Utilities/__init__.py
index a9a74ff3..454b31ff 100644
--- a/RNS/Utilities/__init__.py
+++ b/RNS/Utilities/__init__.py
@@ -33,5 +33,7 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
\ No newline at end of file
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
\ No newline at end of file
diff --git a/RNS/Utilities/rngit/__init__.py b/RNS/Utilities/rngit/__init__.py
index 56fbb793..2a04167a 100644
--- a/RNS/Utilities/rngit/__init__.py
+++ b/RNS/Utilities/rngit/__init__.py
@@ -35,5 +35,7 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
\ No newline at end of file
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
\ No newline at end of file
diff --git a/RNS/Utilities/rnsh/__init__.py b/RNS/Utilities/rnsh/__init__.py
index fbf23102..a528c782 100644
--- a/RNS/Utilities/rnsh/__init__.py
+++ b/RNS/Utilities/rnsh/__init__.py
@@ -41,7 +41,9 @@ module_dir = os.path.dirname(module_abs_filename)
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
def _get_version(): return __version__
diff --git a/RNS/__init__.py b/RNS/__init__.py
index 1c63b3b8..487c0bd4 100755
--- a/RNS/__init__.py
+++ b/RNS/__init__.py
@@ -44,8 +44,10 @@ from functools import partial, wraps
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
import importlib.util
if importlib.util.find_spec("cython"): import cython; compiled = cython.compiled
diff --git a/RNS/vendor/__init__.py b/RNS/vendor/__init__.py
index ccfb449d..7ec88c11 100755
--- a/RNS/vendor/__init__.py
+++ b/RNS/vendor/__init__.py
@@ -3,5 +3,7 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
-modules = py_modules+pyc_modules
-__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
\ No newline at end of file
+so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
+pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
+modules = py_modules+pyc_modules+so_modules+pyd_modules
+__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
\ No newline at end of file
Served by rngit 1.5.2 - Generated in 0.11s